{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 1.0\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matchzoo as mz\n",
    "print('matchzoo version', mz.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classification_task` initialized with metrics [accuracy]\n"
     ]
    }
   ],
   "source": [
    "classification_task = mz.tasks.Classification(num_classes=2)\n",
    "classification_task.metrics = ['acc']\n",
    "print(\"`classification_task` initialized with metrics\", classification_task.metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data loading ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 83032: expected 6 fields, saw 7\\n'\n",
      "b'Skipping line 154657: expected 6 fields, saw 7\\n'\n",
      "b'Skipping line 323916: expected 6 fields, saw 7\\n'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n"
     ]
    }
   ],
   "source": [
    "print('data loading ...')\n",
    "train_pack_raw = mz.datasets.quora_qp.load_data('train', task=classification_task)\n",
    "dev_pack_raw = mz.datasets.quora_qp.load_data('dev', task=classification_task)\n",
    "test_pack_raw = mz.datasets.quora_qp.load_data('test', task=classification_task)\n",
    "print('data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = mz.models.ESIM.get_default_preprocessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 266106/266106 [00:38<00:00, 6984.81it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 273121/273121 [00:36<00:00, 7382.63it/s]\n",
      "Processing text_right with append: 100%|██████████| 273121/273121 [00:00<00:00, 818544.12it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 273121/273121 [00:01<00:00, 242638.93it/s]\n",
      "Processing text_right with transform: 100%|██████████| 273121/273121 [00:01<00:00, 204936.26it/s]\n",
      "Processing text_left with extend: 100%|██████████| 266106/266106 [00:00<00:00, 920048.68it/s]\n",
      "Processing text_right with extend: 100%|██████████| 273121/273121 [00:00<00:00, 983875.24it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 6022311/6022311 [00:01<00:00, 3638545.75it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 266106/266106 [00:34<00:00, 7678.60it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 273121/273121 [00:36<00:00, 7461.54it/s]\n",
      "Processing text_right with transform: 100%|██████████| 273121/273121 [00:01<00:00, 147557.50it/s]\n",
      "Processing text_left with transform: 100%|██████████| 266106/266106 [00:01<00:00, 169375.42it/s]\n",
      "Processing text_right with transform: 100%|██████████| 273121/273121 [00:01<00:00, 179874.07it/s]\n",
      "Processing length_left with len: 100%|██████████| 266106/266106 [00:00<00:00, 953059.14it/s] \n",
      "Processing length_right with len: 100%|██████████| 273121/273121 [00:00<00:00, 960189.79it/s] \n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 37693/37693 [00:04<00:00, 8180.50it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 37655/37655 [00:04<00:00, 8027.78it/s]\n",
      "Processing text_right with transform: 100%|██████████| 37655/37655 [00:00<00:00, 225085.64it/s]\n",
      "Processing text_left with transform: 100%|██████████| 37693/37693 [00:00<00:00, 211651.30it/s]\n",
      "Processing text_right with transform: 100%|██████████| 37655/37655 [00:00<00:00, 75613.66it/s]\n",
      "Processing length_left with len: 100%|██████████| 37693/37693 [00:00<00:00, 975731.98it/s]\n",
      "Processing length_right with len: 100%|██████████| 37655/37655 [00:00<00:00, 1005011.28it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7faef87e7da0>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7faed60050f0>,\n",
       " 'vocab_size': 78764,\n",
       " 'embedding_input_dim': 78764}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=100)\n",
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "embedding_matrix = glove_embedding.build_matrix(term_index)\n",
    "l2_norm = np.sqrt((embedding_matrix * embedding_matrix).sum(axis=1))\n",
    "embedding_matrix = embedding_matrix / l2_norm[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = mz.dataloader.Dataset(\n",
    "    data_pack=train_pack_processed,\n",
    "    mode='point'\n",
    ")\n",
    "devset = mz.dataloader.Dataset(\n",
    "    data_pack=dev_pack_processed,\n",
    "    mode='point'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "padding_callback = mz.models.ESIM.get_default_padding_callback()\n",
    "\n",
    "trainloader = mz.dataloader.DataLoader(\n",
    "    dataset=trainset,\n",
    "    batch_size=40,\n",
    "    stage='train',\n",
    "    sort=False,\n",
    "    callback=padding_callback\n",
    ")\n",
    "devloader = mz.dataloader.DataLoader(\n",
    "    dataset=devset,\n",
    "    batch_size=40,\n",
    "    stage='dev',\n",
    "    sort=False,\n",
    "    callback=padding_callback\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ESIM(\n",
      "  (embedding): Embedding(78764, 100, padding_idx=0)\n",
      "  (rnn_dropout): RNNDropout(p=0.2, inplace=False)\n",
      "  (input_encoding): StackedBRNN(\n",
      "    (rnns): ModuleList(\n",
      "      (0): LSTM(100, 100, bidirectional=True)\n",
      "    )\n",
      "  )\n",
      "  (attention): BidirectionalAttention()\n",
      "  (projection): Sequential(\n",
      "    (0): Linear(in_features=800, out_features=200, bias=True)\n",
      "    (1): ReLU()\n",
      "  )\n",
      "  (composition): StackedBRNN(\n",
      "    (rnns): ModuleList(\n",
      "      (0): LSTM(200, 100, bidirectional=True)\n",
      "    )\n",
      "  )\n",
      "  (classification): Sequential(\n",
      "    (0): Dropout(p=0.2, inplace=False)\n",
      "    (1): Linear(in_features=800, out_features=200, bias=True)\n",
      "    (2): Tanh()\n",
      "    (3): Dropout(p=0.2, inplace=False)\n",
      "  )\n",
      "  (out): Linear(in_features=200, out_features=2, bias=True)\n",
      ")\n",
      "Trainable params:  8600402\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.ESIM()\n",
    "\n",
    "model.params['task'] = classification_task\n",
    "model.params['embedding'] = embedding_matrix\n",
    "model.params['mask_value'] = 0\n",
    "model.params['dropout'] = 0.2\n",
    "model.params['hidden_size'] = 200\n",
    "model.params['lstm_layer'] = 1\n",
    "\n",
    "model.build()\n",
    "\n",
    "print(model)\n",
    "print('Trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(),lr=1e-5)\n",
    "\n",
    "trainer = mz.trainers.Trainer(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    trainloader=trainloader,\n",
    "    validloader=devloader,\n",
    "    validate_interval=None,\n",
    "    epochs=5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2a34aa91d0f45bf90a4bfb9126acdeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9079), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-9079 Loss-0.609]:\n",
      "  Validation: accuracy: 0.7135\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c997c9b116f4eec8a9a73d365091072",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9079), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-18158 Loss-0.555]:\n",
      "  Validation: accuracy: 0.7288\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd5143dfe8554455a255946f93aaf508",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9079), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-27237 Loss-0.536]:\n",
      "  Validation: accuracy: 0.7375\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69e4186f53cd4b4698839525db4309df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9079), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-36316 Loss-0.524]:\n",
      "  Validation: accuracy: 0.744\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7989102303a428898d74e4e6c406898",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9079), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-45395 Loss-0.515]:\n",
      "  Validation: accuracy: 0.7494\n",
      "\n",
      "Cost time: 27346.551464796066s\n"
     ]
    }
   ],
   "source": [
    "trainer.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
